 1.电商分析之--广告业务之ADS层数据导出(DataX)
   
   步骤：
     (1).在MySQL创建对应的表
     (2).创建配置文件（json）
     (3).执行命令，使用json配置文件；测试
     (4).编写执行脚本（shell）
     (5).shell脚本的测试
   1).MySQL 建表
   drop table if exists dwads.ads_ad_show_place;
create table dwads.ads_ad_show_place(
ad_action tinyint,
hour varchar(2),
place varchar(20),
product_id int,
cnt int,
dt varchar(10)
);
   2).创建配置文件
   /data/lagoudw/script/advertisement/ads_ad_show_place.json

{
   "job":{
	   "setting":{
           "speed":{
               "channel":1
           }
       },
       "content":[
           {
               "reader":{
                   "name":"hdfsreader",
                   "parameter":{
"path":"/user/hive/warehouse/ads.db/ads_ad_show_place/dt=$do_date/*",
                       "defaultFS":"hdfs://linux121:9000",
                       "column":[
                           {
                               "index":0,
                               "type":"string"
                           },
                           {
                               "index":1,
                               "type":"string"
                           },
                           {
                               "index":2,
                               "type":"string"
                           },
						   {
                               "index":3,
                               "type":"string"
                           },
                           {
                               "index":4,
                               "type":"string"
                           },
                           {
                               "type":"string",
                               "value":"$do_date"
                           }
				        ],
					    "fileType":"text",
                        "encoding":"UTF-8",
                        "fieldDelimiter":","
					}
                },
				"writer":{
                    "name":"mysqlwriter",
                    "parameter":{
                        "writeMode":"insert",
                        "username":"hive",
                        "password":"12345678",
                        "column":[
                            "ad_action",
                            "hour",
                            "place",
                            "product_id",
                            "cnt",
                            "dt"
                        ],
                        "preSql":[
                            "delete from ads_ad_show_place where dt='$do_date'"
                        ],
						"connection":[
                            {
"jdbcUrl":"jdbc:mysql://linux123:3306/dwads?useUnicode=true&characterEncoding=utf-8",
                                "table":[
                                    "ads_ad_show_place"
                                 ]
                            }
						]
                    }
                }
            }
         ]
    }
}
   3).执行命令(测试)
   python /data/modules/datax/bin/datax.py -p "-Ddo_date=2020-07-30" 
/data/lagoudw/script/advertisement/ads_ad_show_place.json
   
   select count(*) from ads_ad_show_place;
   TRUNCATE TABLE ads_ad_show_place;
   4).编写脚本
   /data/lagoudw/script/advertisement/ads_ad_show_place.sh

#!/bin/bash

source /etc/profile
JSON=/data/lagoudw/script

if [ -n "$1" ] ;then
    do_date=$1
else
    do_date=`date -d "-1 day" +%F`
fi

python $DATAX_HOME/bin/datax.py -p "-Ddo_date=$do_date" $JSON/advertisement/ads_ad_show_place.json
   5).执行脚本
   sh /data/lagoudw/script/advertisement/ads_ad_show_place.sh 2020-07-30
 
 2.高仿日志数据测试
   
   1).数据采集
      1000W 左右日活用户
      按 30 条日志 / 人天，合计3亿条事件日志
      每条日志 650 字节 左右
      总数据量大概在180G
      采集数据时间约2.5 小时
   (1).清理环境
   (2).启动flume
   nohup flume-ng agent --conf /opt/lagou/servers/flume-1.9/conf --conf-file
 /data/lagoudw/conf/flume-log2hdfs4.conf -name a1 -Dflume.root.logger=INFO,console &
   (3).写日志
   cd /data/lagoudw/jars
   
   nohup java -cp data-generator-1.1-SNAPSHOT-jar-withdependencies.
jar com.lagou.ecommerce.AppEvent 300000000 
2020-08-03 > /data/lagoudw/logs/event/eventlog0803.log &
   2).执行脚本
   sh ods_load_event_log.sh 2020-08-03
   sh dwd_load_event_log.sh 2020-08-03
   sh dwd_load_ad_log.sh 2020-08-03
   
   sh ads_load_ad_show.sh 2020-08-03
   sh ads_load_ad_show_rate.sh 2020-08-03
   sh ads_load_ad_show_page.sh 2020-08-03
   sh ads_load_ad_show_page_window.sh 2020-08-03
 